#!/bin/bash

echo "================`date`======================================"

bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin"; pwd`

phone='13811063282'
hostname=`hostname`

. /etc/profile
. ~/.bashrc

function sendMsg(){
  for number in $phone
  do 
    echo $1 " don't exists in $2"
    wall $1 " don't exists in $2"
    gsmsend-script $number@$1 " don't exists in $2"
  done
}

function monitor(){
  for i in 1 2 3
  do
    count=`/usr/local/bin/pdsh -R ssh -w $1 ps aux | grep java | grep $2 | grep -v grep | wc -l`
	if [ $count -eq 0 ]; then
	  echo "$2 not foundi $i times in $1"
	  sleep 5
	else
	  break
	fi
  done

#echo "count: $count"
    if [ $count -eq 0 ]; then
      sendMsg $2 $1
      case $2 in
        namenode | zkfc | datanode | journalnode)
          /usr/local/bin/pdsh -R ssh -w $1 ${HADOOP_HOME}/sbin/hadoop-daemon.sh start $2
        ;;
        resourcemanager | nodemanager | proxyserver)
          /usr/local/bin/pdsh -R ssh -w $1 ${HADOOP_HOME}/sbin/yarn-daemon.sh start $2
        ;;
        historyserver)
          /usr/local/bin/pdsh -R ssh -w $1 ${HADOOP_HOME}/sbin/mr-jobhistory-daemon.sh start $2
        ;;
        *)
          echo "invalid args: $1 $2, ignore it"
        ;;
      esac
    else
      echo "$2 in $1 is ok"
    fi
}

nns='nj03-game-m22dianquan59.nj03.baidu.com nj03-game-m22dianquan58.nj03.baidu.com'
jns='nj03-game-m22dianquan59.nj03.baidu.com nj03-game-m22dianquan58.nj03.baidu.com nj03-game-m22dianquan56.nj03.baidu.com'
rmn='nj03-game-m22dianquan59.nj03.baidu.com'
hs='nj03-game-m22dianquan59.nj03.baidu.com'

#namenode zkfc
for nn in $nns
do
  monitor $nn namenode
  monitor $nn zkfc
done

#journalnode
for jn in $jns
do
   monitor $jn journalnode 
done

#datanode nodemanager
while read line
do
  monitor ${line} datanode
  monitor ${line} nodemanager 
done < ${bin}/etc/hadoop/slaves

#resourcemanager
monitor $rmn resourcemanager

#historyserver
monitor $hs historyserver


